Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Majority voting appears to work

Andrew Cantino 11 years ago
parent
0120ccb9c1
commit
1e8dcc65f0
2 changed files with 63 additions and 6 deletions
Show Diff Stats
  1. 10 3
      app/models/agents/human_task_agent.rb
  2. 53 3
      spec/models/agents/human_task_agent_spec.rb

+ 10 - 3
app/models/agents/human_task_agent.rb
View File 

@@ -167,18 +167,25 @@ module Agents
167 167 
         log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
168 168 
         if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
169 169 
           if options[:take_majority] == "true"
170 
+            counts = {}
170 171 
             options[:hit][:questions].each do |question|
171 
-              counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
172 
+              question_counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
172 173 
               assignments.each do |assignment|
173 174 
                 answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
174 175 
                 answer = answers[question[:key]]
175 
-                counts[answer] += 1
176 
+                question_counts[answer] += 1
176 177 
               end
178 
+              counts[question[:key]] = question_counts
177 179 
             end
180 
+            majority_answer = counts.inject({}) do |memo, (key, question_counts)|
181 
+              memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
182 
+              memo
183 
+            end
184 
+            event = create_event :payload => { :answers => assignments.map(&:answers), :counts => counts, :majority_answer => majority_answer }
178 185 
           else
179 186 
             event = create_event :payload => { :answers => assignments.map(&:answers) }
180 
-            log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
181 187 
           end
188 
+          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
182 189
183 190 
           assignments.each(&:approve!)
184 191

+ 53 - 3
spec/models/agents/human_task_agent_spec.rb
View File 

@@ -12,6 +12,17 @@ describe Agents::HumanTaskAgent do
12 12 
     @event.payload = { :foo => { "bar" => { :baz => "a2b" } },
13 13 
                        :name => "Joe" }
14 14 
     @event.id = 345
15 
+
16 
+    @checker.should be_valid
17 
+  end
18 
+
19 
+  describe "validations" do
20 
+    it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
21 
+      @checker.options[:take_majority] = "true"
22 
+      @checker.should_not be_valid
23 
+      @checker.options[:hit][:questions][1][:type] = "selection"
24 
+      @checker.should be_valid
25 
+    end
15 26 
   end
16 27
17 28 
   describe "when 'trigger_on' is set to 'schedule'" do
 
@@ -223,12 +234,51 @@ describe Agents::HumanTaskAgent do
223 234 
     end
224 235
225 236 
     describe "taking majority votes" do
226 
-      it "should only be valid when all questions are of type 'selection'" do
227 
-
237 
+      before do
238 
+        @checker.options[:take_majority] = "true"
239 
+        @checker.options[:hit][:questions][1] = {
240 
+          :type => "selection",
241 
+          :key => "age_range",
242 
+          :name => "Age Range",
243 
+          :required => "true",
244 
+          :question => "Please select your age range:",
245 
+          :selections =>
246 
+            [
247 
+              { :key => "<50", :text => "50 years old or younger" },
248 
+              { :key => ">50", :text => "Over 50 years old" }
249 
+            ]
250 
+        }
228 251 
       end
229 252
230 253 
       it "should take the majority votes of all questions" do
231 
-
254 
+        @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
255 
+        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
256 
+        assignments = [
257 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "age_range"=>"<50"}),
258 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "age_range"=>">50"}),
259 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"}),
260 
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "age_range"=>">50"})
261 
+        ]
262 
+        hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
263 
+        mock(RTurk::Hit).new("JH3132836336DHG") { hit }
264 
+
265 
+        lambda {
266 
+          @checker.send :review_hits
267 
+        }.should change { Event.count }.by(1)
268 
+
269 
+        assignments.all? {|a| a.approved == true }.should be_true
270 
+
271 
+        @checker.events.last.payload[:answers].should == [
272 
+          { :sentiment => "sad", :age_range => "<50" },
273 
+          { :sentiment => "neutral", :age_range => ">50" },
274 
+          { :sentiment => "happy", :age_range => ">50" },
275 
+          { :sentiment => "happy", :age_range => ">50" }
276 
+        ]
277 
+
278 
+        @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } }
279 
+        @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" }
280 
+
281 
+        @checker.memory[:hits].should == {}
232 282 
       end
233 283 
     end
234 284 
   end